# -*- coding: utf-8 -*-
# @Time: 2025/7/25 14:43
# @Author: foxhuty
# @File: crawling_douban_movie_250.py
import requests
import re
import pandas as pd

cookies = {
    'bid': 'rgoppNNZ-aY',
    '_pk_ref.100001.4cf6': '%5B%22%22%2C%22%22%2C1753423497%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3Dw5GHlpes260fVxw2hdYVHUWb629lTaS3sD09QrHLbxxIXDkR25oQGUysvTRrWEyR%26wd%3D%26eqid%3Db15ff9ae0008d3ee0000000668831e83%22%5D',
    '_pk_id.100001.4cf6': 'c7a402e1c6b99fd1.1753423497.',
    'ap_v': '0,6.0',
    '__utma': '30149280.704574464.1753423498.1753423498.1753423498.1',
    '__utmc': '30149280',
    '__utmz': '30149280.1753423498.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic',
    '__utma': '223695111.558633792.1753423498.1753423498.1753423498.1',
    '__utmc': '223695111',
    '__utmz': '223695111.1753423498.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic',
    '__yadk_uid': 'YxsXbFMwDFowemiLZHz2zme9I4dXZBGt',
}

headers = {
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
    'accept-language': 'zh-CN,zh;q=0.9',
    'cache-control': 'max-age=0',
    'priority': 'u=0, i',
    'referer': 'https://www.baidu.com/link?url=w5GHlpes260fVxw2hdYVHUWb629lTaS3sD09QrHLbxxIXDkR25oQGUysvTRrWEyR&wd=&eqid=b15ff9ae0008d3ee0000000668831e83',
    'sec-ch-ua': '"Not)A;Brand";v="8", "Chromium";v="138", "Google Chrome";v="138"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"Windows"',
    'sec-fetch-dest': 'document',
    'sec-fetch-mode': 'navigate',
    'sec-fetch-site': 'cross-site',
    'sec-fetch-user': '?1',
    'upgrade-insecure-requests': '1',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36',
    # 'cookie': 'bid=rgoppNNZ-aY; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1753423497%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3Dw5GHlpes260fVxw2hdYVHUWb629lTaS3sD09QrHLbxxIXDkR25oQGUysvTRrWEyR%26wd%3D%26eqid%3Db15ff9ae0008d3ee0000000668831e83%22%5D; _pk_id.100001.4cf6=c7a402e1c6b99fd1.1753423497.; ap_v=0,6.0; __utma=30149280.704574464.1753423498.1753423498.1753423498.1; __utmc=30149280; __utmz=30149280.1753423498.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __utma=223695111.558633792.1753423498.1753423498.1753423498.1; __utmc=223695111; __utmz=223695111.1753423498.1.1.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; __yadk_uid=YxsXbFMwDFowemiLZHz2zme9I4dXZBGt',
}


# url = 'https://movie.douban.com/top250?start=0&filter='


def get_html(url):
    response = requests.get(url, headers=headers)
    response.encoding = 'utf-8'
    html = response.text
    # print(html)
    return html


def get_douban_data(html):
    obj = re.compile(
        r'<div class="item">.*?<span class="title">(?P<name>.*?)</span>.*?导演:(?P<director>.*?)&nbsp.*?主演:(?P<actor>.*?)...<br>(?P<year>.*?)&nbsp.*?<span class="rating_num"'
        r' property="v:average">(?P<score>.*?)</span>.*?<span>(?P<number>.*?)人评价</span>',
        re.S)
    result = obj.finditer(html)
    dict_list = []
    for item in result:
        douban_dict = item.groupdict()
        douban_dict['year'] = douban_dict['year'].strip()
        print(douban_dict)
        dict_list.append(douban_dict)
    df = pd.DataFrame(data=dict_list)
    return df


def main(pages):
    total_list = []
    for page in range(pages):
        url = f'https://movie.douban.com/top250?start={page * 25}&filter='
        html = get_html(url)
        data = get_douban_data(html)
        total_list.append(data)
        print(f'第{page + 1}页数据获取完成'.center(100, '*'))
    total_df = pd.concat(total_list, axis=0)
    total_df.reset_index(drop=True, inplace=True)
    total_df.index += 1
    # total_df.to_excel('douban_movie_250.xlsx', index=True, index_label='序号')
    # total_df.to_csv('douban_movie_250.csv', index=True, index_label='序号', encoding='utf-8-sig')


if __name__ == '__main__':
    main(10)
